#include "../include/split_tool.hpp"

#include <fstream>

using std::ifstream;

namespace gcj
{

SplitTool::SplitTool()
: _jieba("../cppjieba/dict/jieba.dict.utf8"
        , "../cppjieba/dict/hmm_model.utf8"
        , "../cppjieba/dict/user.dict.utf8"
        , "../cppjieba/dict/idf.utf8"
        , "../cppjieba/dict/stop_words.utf8")
{
    ifstream ifs("../res/stop-words-list/stop_words_zh.txt");
    string word;
    while(ifs >> word) {
        _stopWords.insert(word);
    }
    _stopWords.insert(" ");
    _stopWords.insert("\n");
}

vector<string> SplitTool::cut(const string str) {
    //分词器，使用了jieba的CutForSearch方法
    vector<string> words;  
    vector<string> res;     //用来存放结果

    _jieba.CutForSearch(str, words);

    for(auto & word : words) {
        if(_stopWords.find(word) != _stopWords.end()) {
            //如果是停止词，跳过
            continue;
        }
        else {
            res.push_back(word);
        }
    }

    return res;
}

set<string> & SplitTool::getStopWords() {
    return _stopWords;
}

}//end of namespace gcj

